#ifndef _CHARSET_H_
#define _CHARSET_H_

#define CODE_MAP_SIZE	(256)

/*
 *	-------------------------------------------------
 *	0| 1 - 7 | 8 - 15 | 16 - 23 | 24 - 27 | 28 - 31 |
 *	-------------------------------------------------
 *  z: 0-7 byte contains both char tag and word tag, z is the key to look for tag kind
 *  1-27 contains word tag
 *	28-31 only special word
 */

/* ascii char tag saved in code-map */
#define _u_		(0x00)			//unknown									|
#define _z_		(0x01)			//zh is the key on char tag or word tag		|
#define _l_		(0x02)			//\r\n										|
#define _d_		(0x04)			//delimiter: blank tab and all punctuation	|
#define _s_		(0x08)			//symbol									|
#define _m_		(0x10)			//math number								|
#define _e_		(0x20)			//english									|
#define _n_		(_z_ | _m_) 	//pure zh num both as char tag and word tag	|
#define _f_		(_z_ | _e_)		//zh english both as char tag and word tag	|

/* zh word tag saved in dat, z is the key */
#define _N_		(0x00000002)	//noun unuse now
#define _V_		(0x00000004)	//verb unuse now
#define _S_		(0x00000008)	//make sure S = s
#define _M_		(0x00000010)	//make sure M = m
#define _E_		(0x00000020)	//make sure E = e
#define _P_		(0x00000040)	//prepositional now only for oov correction
#define _Q_		(0x00000080)	//quantity now only for num oov
//more ... ...

/* extra zh word tag saved in dat, z is not necessary */
#define _NRPRE_	(0x10000000)	//zh first name, hunman name prefix			|
#define _NSSUF_	(0x20000000)	//zh location name suffix					|
#define _NRRGH_	(0x40000000)	//zh human name right edge					|
#define _FORN_	(0x80000000)	//foreign name chars						|


int init_codemap();
unsigned char ctag(unsigned short code);
int utf8decode(const char *p, unsigned short *codep);
int free_codemap();

#endif
